#coding=utf-8
import requests
import time
import csv
import pandas as pd
import logging
from apscheduler.schedulers.blocking import BlockingScheduler
import re
from lxml import etree
import psutil
import os


class solreSpider(object):
    def __init__(self):
        self.headers = {
        'Host': 'www.solrenview.com',
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
        'Accept-Encoding': 'gzip, deflate, br',
        'Accept-Language': 'zh-CN,zh;q=0.8',
        'Connection': 'keep-alive',
        }
        self.urlbase = 'https://www.solrenview.com/cgi-bin/cgihandler.cgi?'
        self.query1 = ['&view=0,0,0,0', '&view=1,0,0,0', '&view=2,0,0,0', '&view=3,0,0,0', '&view=4,0,0,0']
        self.query2 = ['&cond=site_ID=66','&cond=site_ID=43','&cond=site_ID=44']
        self.content1 =  {'Inverter':[],
                         'AC_Energy_Today':[],
                         'AC_Energy_LifeTime':[],
                         'AC_Energy_TimeStamp':[],
                         'AC_Power_Today_Max_Reading':[],
                         'AC_Power_Today_Last_Reading': [],
                         'AC_Voltage_Today_Last_Reading_a': [],
                         'AC_Voltage_Today_Last_Reading_b': [],
                         'AC_Voltage_Today_Last_Reading_c': [],
                         'AC_Voltage_Today_TimeStamp': [],
                         'AC_Current_Today_Max_Reading': [],
                         'AC_Current_Today_Last_Reading': [],
                         'DC_Voltage_Today_Last_Reading': [],
                         'DC_Voltage_Today_TimeStamp': [],
                         'siteName': [],
                         'status': [],
                         'Energy_Generated_Today': [],
                         'Lifetime_Energy_Generated':[],
                         'Ambient_Temperature': [],
                         'Module_Temperature': [],
                         'Emission_Offset': [],
                         'System_AC_Power_Now': [],
                         'wind': [],
                         'Irradiance': [],
                                                         }
        self.content =  {'Inverter':[],
                         'AC_Energy_Today':[],
                         'AC_Energy_LifeTime':[],
                         'AC_Energy_TimeStamp':[],
                         'AC_Power_Today_Max_Reading':[],
                         'AC_Power_Today_Last_Reading': [],
                         'AC_Voltage_Today_Last_Reading_a': [],
                         'AC_Voltage_Today_Last_Reading_b': [],
                         'AC_Voltage_Today_Last_Reading_c': [],
                         'AC_Voltage_Today_TimeStamp': [],
                         'AC_Current_Today_Max_Reading': [],
                         'AC_Current_Today_Last_Reading': [],
                         'DC_Voltage_Today_Last_Reading': [],
                         'DC_Voltage_Today_TimeStamp': [],
                         'siteName': [],
                         'status': [],
                         'Energy_Generated_Today': [],
                         'Lifetime_Energy_Generated':[],
                         'Ambient_Temperature': [],
                         'Module_Temperature': [],
                         'Emission_Offset': [],
                         'System_AC_Power_Now': [],
                         'wind': [],
                         'Irradiance': [],
                                                         }

    def parse1(self):
        currentTime = time.time()-86400
        currentTime = time.localtime(currentTime)
        currentTime = time.strftime('%Y-%m-%d-', currentTime)
        for i in self.query2:
            s = requests.Session()
            time.sleep(2)
            r = s.get(url=self.urlbase+'&sort=pvi_IDs'+i,headers =self.headers)
            tree = etree.HTML(r.text)
            siteName = re.search('- (.*?) -',r.text).group(1)
            status = tree.xpath('//*[@id="StatusBox"]/b/text()')[0]
            Energy_Generated_Today = re.search('Energy Generated Today.*?([0-9]+)<',r.text).group(1)
            Lifetime_Energy_Generated = re.search('Lifetime Energy Generated.*?([0-9]+)<',r.text).group(1)
            Emission_Offset = re.search('Emission Offset.*?>([0-9]+)<',r.text).group(1)
            System_AC_Power_Now = re.search('System AC Power Now.*?([0-9]+)<',r.text).group(1)
            Irradiance = re.search('Irradiance.*?([0-9]+)<',r.text)
            Ambient_Temperature = re.search('Ambient Temperature.*?([0-9]+)<',r.text)
            Module_Temperature = re.search('Module Temperature.*?([0-9]+)<',r.text)
            wind = re.search('Wind .*?>([0-9]+\.[0-9]+)<',r.text)
            # for j in self.query1:
            r2 = s.get(url=self.urlbase + self.query1[0] + i,headers =self.headers)
            r3 = s.get(url=self.urlbase + self.query1[1] + i,headers =self.headers)
            r4 = s.get(url=self.urlbase + self.query1[2] + i, headers=self.headers)
            r5 = s.get(url=self.urlbase + self.query1[3] + i, headers=self.headers)
            r6 = s.get(url=self.urlbase + self.query1[4] + i, headers=self.headers)
            tree2 = etree.HTML(r2.text)
            Inverter = tree2.xpath('//*[@id="row1" or @id="row2" or @id="row3" or @id="row4"]/span/text()')
            AC_Energy_Today = re.findall('rdiv.*?>([0-9]+\.[0-9])',r2.text)
            AC_Energy_LifeTime = re.findall('>([0-9]+\.[0-9]) kWh<',r2.text)
            AC_Energy_TimeStamp = re.findall('>\[(.*?:.*?:.*?)\]<',r2.text)
            AC_Power_Today_Max_Reading = re.findall('WrPeakDataCell\(\'p.*?\'\,\'(.*?).W\'',r3.text)
            AC_Power_Today_Last_Reading = re.findall('WrPeakDataCell\(\'\'\,\'([0-9]+).*?\'',r3.text)
            AC_Voltage_Today_Last_Reading = re.findall('>([0-9]+/[0-9]+/[0-9]+).*?<',r4.text)
            # AC_Voltage_Today_Last_Reading_b = re.findall('>([0-9]+/[0-9]+/[0-9]+).*?<',r4.text).split()
            # AC_Voltage_Today_Last_Reading_c = re.findall('>([0-9]+/[0-9]+/[0-9]+).*?<',r4.text).split()
            AC_Voltage_Today_TimeStamp = re.findall('>\[(.*?:.*?:.*?)\]<',r4.text)
            AC_Current_Today_Max_Reading = re.findall('WrPeakDataCell\(\'p.*?\'([0-9]+) ',r5.text)
            AC_Current_Today_Last_Reading = re.findall('WrPeakDataCell\(\'\'\,\'([0-9]+) ',r5.text)
            DC_Voltage_Today_Last_Reading = re.findall('>([0-9]+) V<',r6.text)
            DC_Voltage_Today_TimeStamp = re.findall('>\[(.*?:.*?:.*?)\]<',r6.text)
            for j in range(len(Inverter)):
                time.sleep(2)
                self.content['siteName'].append(siteName)
                self.content['status'].append(status)
                self.content['Energy_Generated_Today'].append(Energy_Generated_Today)
                self.content['Lifetime_Energy_Generated'].append(Lifetime_Energy_Generated)
                self.content['Emission_Offset'].append(Emission_Offset)
                self.content['System_AC_Power_Now'].append(System_AC_Power_Now)
                self.content['Inverter'].append(Inverter[j])
                if AC_Energy_Today:
                    if wind:
                        self.content['Irradiance'].append(Irradiance.group(1))
                        self.content['Ambient_Temperature'].append(Ambient_Temperature.group(1))
                        self.content['Module_Temperature'].append(Module_Temperature.group(1))
                        self.content['wind'].append(wind.group(1))
                    else:
                        self.content['Irradiance'].append('0')
                        self.content['Ambient_Temperature'].append('0')
                        self.content['Module_Temperature'].append('0')
                        self.content['wind'].append('0')
                    self.content['AC_Energy_Today'].append(AC_Energy_Today[j])
                    self.content['AC_Energy_LifeTime'].append(AC_Energy_LifeTime[j])
                    self.content['AC_Energy_TimeStamp'].append(currentTime + AC_Energy_TimeStamp[j])
                    self.content['AC_Power_Today_Max_Reading'].append(AC_Power_Today_Max_Reading[j])
                    self.content['AC_Power_Today_Last_Reading'].append(AC_Power_Today_Last_Reading[j])
                    self.content['AC_Voltage_Today_Last_Reading_a'].append(AC_Voltage_Today_Last_Reading[j].split('/')[0])
                    self.content['AC_Voltage_Today_Last_Reading_b'].append(AC_Voltage_Today_Last_Reading[j].split('/')[1])
                    self.content['AC_Voltage_Today_Last_Reading_c'].append(AC_Voltage_Today_Last_Reading[j].split('/')[2])
                    self.content['AC_Voltage_Today_TimeStamp'].append(currentTime + AC_Voltage_Today_TimeStamp[j])
                    self.content['AC_Current_Today_Max_Reading'].append(AC_Current_Today_Max_Reading[j])
                    self.content['AC_Current_Today_Last_Reading'].append(AC_Current_Today_Last_Reading[j])
                    self.content['DC_Voltage_Today_Last_Reading'].append(DC_Voltage_Today_Last_Reading[j])
                    self.content['DC_Voltage_Today_TimeStamp'].append(currentTime + DC_Voltage_Today_TimeStamp[j])
                else:
                    if wind:
                        self.content['Irradiance'].append(Irradiance.group(1))
                        self.content['Ambient_Temperature'].append(Ambient_Temperature.group(1))
                        self.content['Module_Temperature'].append(Module_Temperature.group(1))
                        self.content['wind'].append(wind.group(1))
                    else:
                        self.content['Irradiance'].append('0')
                        self.content['Ambient_Temperature'].append('0')
                        self.content['Module_Temperature'].append('0')
                        self.content['wind'].append('0')
                    self.content['AC_Energy_Today'].append(0)
                    self.content['AC_Energy_LifeTime'].append(0)
                    self.content['AC_Energy_TimeStamp'].append(0)
                    self.content['AC_Power_Today_Max_Reading'].append(0)
                    self.content['AC_Power_Today_Last_Reading'].append(0)
                    self.content['AC_Voltage_Today_Last_Reading_a'].append(0)
                    self.content['AC_Voltage_Today_Last_Reading_b'].append(0)
                    self.content['AC_Voltage_Today_Last_Reading_c'].append(0)
                    self.content['AC_Voltage_Today_TimeStamp'].append(0)
                    self.content['AC_Current_Today_Max_Reading'].append(0)
                    self.content['AC_Current_Today_Last_Reading'].append(0)
                    self.content['DC_Voltage_Today_Last_Reading'].append(0)
                    self.content['DC_Voltage_Today_TimeStamp'].append(0)
            s.close()

    def writeCsv(self,data):
        currentTime = time.time()
        currentTime = time.localtime(currentTime)
        currentTime = time.strftime('%Y-%m-%d,%H-%M-%S', currentTime)
        df = pd.DataFrame(data=data,columns=data.keys())
        df.to_csv('F:/python/solren/solren-%s.csv'%str(currentTime),index=False)

    def start(self):
        logging.basicConfig(filename='solrenSpider.log', filemode='w', level=logging.INFO)
        start = time.time()
        self.parse1()
        self.writeCsv(data=self.content)
        self.content=self.content1
        end = time.time()
        logging.info('consume time:%s'%str(end-start))
        print 'script run time:%s'%str(end-start)

if __name__ == "__main__":
    # sched =BlockingScheduler()
    # sched.add_job(solreSpider().start,'interval',minutes = 15,start_date ='2017-07-21 15:31:00',end_date = '2017-07-21 16:50:00')
    # sched.start()
    while True:
        try:
            solreSpider().start()
            time.sleep(900)
        except BaseException,e:
            print 'this exception is:',e
            print 'cannot request'
            time.sleep(60)
        p1 = psutil.Process(os.getpid())

        print ('直接打印内存占用： ' + (str)(psutil.virtual_memory))

        # 打印内存的占用率
        print ('获取内存占用率： ' + (str)(psutil.virtual_memory().percent) + '%')

        # 本机cpu的总占用率
        print ('打印本机cpu占用率： ' + (str)(psutil.cpu_percent(0)) + '%')

        # 该进程所占cpu的使用率
        print (" 打印该进程CPU占用率: " + (str)(p1.cpu_percent(None)) + "%")

        # 直接打印进程所占内存占用率
        print (p1.memory_percent)

        # 格式化后显示的进程内存占用率
        print "percent: %.2f%%" % (p1.memory_percent())



    # solreSpider().start()